using System;
using System.Text;
using System.IO;
using System.Collections;
using System.Net;

namespace Slaks.Web.Parser
{
	/// <summary>
	/// <para>
	/// Slaks.DocumentHtmlParser implements parsing mechanism of the html file.
	/// The parsing mechanism allows parsing html structures such as FORM, TABLE, text and url link extraction
	/// </para>
	/// <para>
	/// The following code demonstrates how to parse the web sourse and get the parsed information into the file "c:\test.txt"
	/// </para>
	/// <code>
	/// TextWriter writer = null;
	/// try
  /// {
	///    Slaks.Web.Parser.DocumentHtmlParser.Utils.WebPageDownload.TimeOut = 10000;
	///    DocumentHtmlParser parser = new DocumentHtmlParser("http://www.ise.bgu.ac.il/courses/mdss/",System.Text.Encoding.Default);
	///    parser.ParseMe();
	///   //writer = Console.Out; can be a console output
  ///
	///    FileStream fs = new FileStream(@"c:\test.txt",
	///    FileMode.Create, FileAccess.Write, FileShare.None);
	///   
	///    writer = new StreamWriter(fs,System.Text.Encoding.UTF8);
  ///	   parser.ReadStructures(writer,parser.RootStructure);
  /// }
  ///	 catch(Exception ex)
  ///  {
  ///	    Console.WriteLine(ex.Message + "\n" + ex.StackTrace );
  ///  }
  ///	 finally
  ///  {
	///     if (writer != null) writer.Close();
  ///  }
	/// </code>
	/// </summary>
	public class DocumentHtmlParser : ParserBasics
	{
		protected enum CharProcessing { Tag, Symbol, Control, Space };

		protected CharProcessing m_charProcessing;

		protected string m_currentWord = String.Empty;

		protected bool m_headerFound = false;

		protected HtmlHeader m_header = null;

		protected HtmlStructure m_structure = null;

    protected int m_structureTotalCounter = 0;

    protected char m_currentChar = ' ';

    protected static bool m_isSaveLinkAsText = false;

	  public event ParserEventHandler ParserLogEvent = null;

		protected DocumentHtmlParser() {}

		public DocumentHtmlParser(string localFileName,Encoding encoding):base(localFileName,encoding)
		{
		}

		public DocumentHtmlParser(string internetPath,string addressLinkLocation,Encoding encoding):base(internetPath,addressLinkLocation,encoding)
		{
		}

    /// <summary>
    /// Main function for Parsing
    /// </summary>
		public bool ParseMe()
		{
		  HtmlStructure.ResetCounter();
			bool isSuccess = false;
			try
			{
        isSuccess = this.GetWebPage();
        ///parse if the page was downloaded successfully
        if (isSuccess)
        {
          this.SkipWhiteSpace();
        
          //skip doctype
          if (this.Compare("<!doctype")) 
          {
            this.SkipUntilPatternFound(">",'>',FindDirection.Right);
            AdvanceIndex();
          }
          m_structure = new HtmlStructure("BODY","/body",0);
          this.ParseStructure(m_structure);
        }
				else
        {
					string message = "Problem occurred while downloading the " + m_fileName; 
					if (ParserLogEvent != null) ParserLogEvent(this,new ParserEventArgs(message,null));
        }
			}
			catch(Exception ex)
			{
				if (ParserLogEvent != null) ParserLogEvent(this,new ParserEventArgs("Parser : " + m_fileName + " : " + ex.Message + Environment.NewLine + ex.StackTrace,null));
				throw ex;
			}

			return isSuccess;
		}
		/// <summary>
		/// parse structures of the html files, including BODY, TABLE, TR, SELECT
		/// </summary>
		protected virtual void ParseStructure(HtmlStructure htmlStructure)
		{
			bool isEndOfSentenceMarker = false;
			try
			{
				
				while (!Eof())
				{
          //if the '<' symbol found
					if (this.Compare('<'))
					{
						//if there is a word in buffer
						if (m_currentWord != String.Empty)
						{
							htmlStructure.AddText(m_currentWord);
							m_currentWord = String.Empty;
						}

						m_charProcessing = CharProcessing.Tag;

						AdvanceIndex();
						//check for commnet <!--...--> tag or <!DOCTYPE> tag
						if (this.Compare('!'))
						{
							if (this.CurrentChar(1) == '-')
								//pattern must be --> but some morrons put not valid comments. so i changed
								//the pattern to >
							  this.SkipUntilPatternFound(">",'>',FindDirection.Right);
							else
								this.SkipUntilPatternFound(">",'>',FindDirection.Right);
						}
						else //in valid HTML after '<' the tag name must come, read it by ReadUntilNonLetter function
						{
							string tag = this.ReadTag();
                
              if (this.CheckHeader(tag)) {}
              else if (this.CheckBody(tag)) {}
              else if (this.SkipTags(tag)) {}
              //else if (this.ImportantTags(tag,htmlStructure,ref isEndOfSentenceMarker))
              //{
              //                  if (isEndOfSentenceMarker)
              //                  {
              //                      htmlStructure.AddText(".");
              //                      isEndOfSentenceMarker = false;
              //                  }
              //}
              else this.StructsToConsider(tag, htmlStructure);

								//if it is compound structure then find the close tag of this structure and return one level up
							if (htmlStructure.CloseTagRepresentation == tag) return;
							else if (tag == "/html") return;
							else 
							{
								this.SkipUntilPatternFound(">",'>',FindDirection.Right);
							}
						}
					}
					else //text and special characters
					{
						if (this.IsSpecialEntityCharacters() == true) //special entity characters, put instead space character
						{
							m_charProcessing = CharProcessing.Space;
							m_currentChar = ' ';
						}
						else if(this.IsSpecialEntityUnicode(ref m_currentChar)) //special unicode characters
						{
							if (char.IsWhiteSpace(m_currentChar)) m_charProcessing = CharProcessing.Space;
							else if (char.IsControl(m_currentChar)) m_charProcessing = CharProcessing.Control;
							else m_charProcessing = CharProcessing.Symbol;

						}
						else //text
						{
							m_currentChar = CurrentChar();

							if (char.IsWhiteSpace(m_currentChar)) m_charProcessing = CharProcessing.Space;
							else if (char.IsControl(m_currentChar)) m_charProcessing = CharProcessing.Control;
							else m_charProcessing = CharProcessing.Symbol;
						}

						//check the characters
						if (m_charProcessing == CharProcessing.Space)
						{
							if (m_currentWord.Length > 0 && !char.IsWhiteSpace(m_currentWord[m_currentWord.Length - 1])) m_currentWord += ' ';
						}
						else if (m_charProcessing == CharProcessing.Control)
						{
							if (m_currentWord.Length > 0)
							{
								htmlStructure.AddText(m_currentWord);
								
								m_currentWord = String.Empty;
							}
						}
						else
						{
							if (IsPunctuationCharacter(m_currentChar)) //some characters that has no meaning or punctuation characters
							{
								if (m_currentWord.Length > 0)
								{
									//if (!char.IsWhiteSpace(m_currentWord[m_currentWord.Length - 1])) m_currentWord= ' ';

									htmlStructure.AddText(m_currentWord);
									m_currentWord = String.Empty;
								}
								//else m_currentWord += ' ';
							}
							else
							{
								m_currentWord += m_currentChar;
							}
						}
					}
					AdvanceIndex();
				}//end of while

				if (m_currentWord.Length > 0)
				{
					htmlStructure.AddText(m_currentWord);
				}

			}
			catch(Exception ex)
			{
				throw ex;
			}
		}


		/// <summary>
		/// check for special entity character occurrence (&nbsp;)
		/// </summary>
		/// <returns>true/false</returns>
		protected bool IsSpecialEntityCharacters()
		{
			if (CurrentChar() == '&')  //the special characters like '&nbsp;'
				//if there is a chance that the character could be a special entity or unicode then
				if (CurrentChar(3) == ';' ||
					CurrentChar(4) == ';' ||
					CurrentChar(5) == ';' ||
					CurrentChar(6) == ';' ||
					CurrentChar(7) == ';')
				{
					//if the character is not unicode but special entity
					if (CurrentChar(1) != '#')
					{
						this.SkipUntilPatternFound(";",';',FindDirection.Right);
						return true;
					}
				}
			return false;
		}

		/// <summary>
		/// Check for unicode character in the form &#1234;
		/// </summary>
		/// <param name="character">the character that is defined by the html unicode format</param>
		/// <returns>true/false</returns>
		protected bool IsSpecialEntityUnicode(ref char character)
		{
			if (CurrentChar() == '&')  //the special characters like '&nbsp;'
				//if there is a chance that the character could be a special entity or unicode then
				if (CurrentChar(3) == ';' ||
					CurrentChar(4) == ';' ||
					CurrentChar(5) == ';' ||
					CurrentChar(6) == ';' ||
					CurrentChar(7) == ';')
				{
					if (CurrentChar(1) == '#')
					{
						string unicode = this.ReadUntilPatternFound(";",';',FindDirection.Right);
						char unicodeChar = DocumentHtmlParser.GetUnicodeChar(unicode);
						if (unicodeChar == 0) return false;

						character = unicodeChar;
						return true;
					}
				}
				return false;
		}

		/// <summary>
		/// Set or Get the basic uri of the web page
		/// </summary>
		public Uri BaseUri
		{
			get { return m_baseUri; }
			//set { m_baseUri = value; }
		}

		public bool IsLocalParsing() 
		{ 
			if (m_source == ReadSource.Local) return true;
			return false; 
		}

		public Encoding GetEncoding() { return m_encoding; }

		public Uri GetAddressLinkLocation() 
		{ 
			return m_addressLinkLocation; 
		}

		/// <summary>
		/// Get the number of structures (BODY, FORM, TABLE) in the html
		/// </summary>
		public int TotalStructuresFound
		{
			get { return HtmlStructure.TotalCounter; }
		}

		/// <summary>
		/// Get Html Header
		/// </summary>
		public HtmlHeader Header
		{
			get { return m_header; }
		}

		/// <summary>
		/// Get Html Root (BODY) Structure
		/// </summary>
		public HtmlStructure RootStructure
		{
			get { return m_structure; }
		}

    /// <summary>
    /// Get the specified structure found in the page by its index (from 1 to number of structures found)
    /// </summary>
    /// <param name="index"></param>
    /// <returns></returns>
		public HtmlStructure GetStructureByIndex(int index)
		{
			if (index > this.TotalStructuresFound || index <= 0) return null;
      return m_structure.GetStructureByIndex(index);
		}

    public static bool IsSaveLinkAsText
    {
      get { return m_isSaveLinkAsText; }
      set { m_isSaveLinkAsText = value; }
    }

    protected bool CheckHeader(string tag)
    {
      #region Header
      
      if (m_headerFound == false)
      {
        if (tag == "/head")	
        {
          m_headerFound = true;
        }
          //title found read it 
        else if (tag == "title")
        {
          if (m_header == null) m_header = new HtmlHeader();
          string title = this.ReadUntilPatternFound("/title>",'/',FindDirection.Right);
          m_header.ExtractTitle(title);
          return true;
        }
          //if meta tag found then read the meta into the header
        else if (tag == "meta") 
        {
          if (m_header == null) m_header = new HtmlHeader();
          string metaname = this.ReadUntilPatternFound(">",'>',FindDirection.Right);
          m_header.ExtractMetaName(metaname);
          return true;
        }
        else if (tag == "style")
        {
          this.SkipUntilPatternFound("/style>",'/',FindDirection.Right);
          return true;
        }
        else if (tag == "link")
        {
          this.SkipUntilPatternFound(">",'>',FindDirection.Right);
          return true;
        }

      }
      #endregion 
      return false;
    }

    protected bool CheckBody(string tag)
    {
      if (tag == "body")
      {
        this.SkipUntilPatternFound(">",'>',FindDirection.Right);
        return true;
      }
      return false;
    }
 
    protected bool SkipTags(string tag)
    {
  		//script tag found -> skip it
			if (tag == "script") 
			{
				this.SkipUntilPatternFound("/script>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "noscript")
			{
				this.SkipUntilPatternFound("/noscript>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "img")
			{
				this.SkipUntilPatternFound(">",'>',FindDirection.Right);
				return true;
			}
			else if (tag == "map") 
			{
				this.SkipUntilPatternFound("/map>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "applet") 
			{
				this.SkipUntilPatternFound("/applet>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "iframe") 
			{
				this.SkipUntilPatternFound("/iframe>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "style")
			{
				this.SkipUntilPatternFound("/style>",'/',FindDirection.Right);
				return true;
			}
			else if (tag == "object")
			{
				this.SkipUntilPatternFound("/object>",'/',FindDirection.Right);
				return true;
			}
      return false;
    }

    protected virtual bool ImportantTags(string tag, HtmlStructure htmlStructure,ref bool isEndOfSentenceMarker)
    {
			isEndOfSentenceMarker = false;
      //check the references
      if (tag == "a") 
      {
        string anchor = this.ReadUntilPatternFound(">",'>',FindDirection.Right);
        bool isHref = HtmlAnchor.IsHref(anchor);
        if (isHref)
        {
          AdvanceIndex();
          string restOfText = this.ReadUntilPatternFound("/a>",'/',FindDirection.Right);
          HtmlAnchor extractedAnchor = HtmlAnchor.ExtractAnchor(anchor,restOfText,this.BaseUri);

					if (extractedAnchor != null) 
					{
						htmlStructure.Anchors.Add(extractedAnchor);
						if (DocumentHtmlParser.IsSaveLinkAsText)
							htmlStructure.AddText(extractedAnchor.Text);
					}
        }
        return true;
      }
      else if (tag == "frame")
      {
        string frameText = this.ReadUntilPatternFound(">",'>',FindDirection.Right);
        HtmlAnchor frame = HtmlAnchor.ExtractFrame(frameText,this.BaseUri);
        if (frame != null)
        {
          htmlStructure.Anchors.Add(frame);
        }
        return true;
      }
			else if (tag == "h1" || tag == "/h1" ||
				       tag == "h2" || tag == "/h2" || 
				       tag == "h3" || tag == "/h3" || 
				       tag == "h4" || tag == "/h4" ||  
				       tag == "h5" || tag == "/h5" || 
			         tag == "h6" || tag == "/h6" ||
				       tag == "hr"  ||
				       tag == "ul" || tag == "/ul" ||
				       tag == "li" || tag == "/li" ||
				       tag == "ol" || tag == "/ol" ||
				       tag == "td" || tag == "/td" ||
				       tag == "tr" || tag == "/tr" ||
				       tag == "th" ||  tag == "/th" || 
				       tag == "tfoot" || tag == "/tfoot" ||
				       tag == "thead" || tag == "/thead" ||
				       tag == "table" || tag == "/table" ||
				       tag == "option" || tag == "/option" ||
				       tag == "select" || tag == "/select" ||
				       tag == "caption" || tag == "/caption" ||
				       tag == "label" || tag == "/label" ||
				       tag == "p" || tag == "/p" ||
							 tag == "fieldset" || tag == "/fieldset" ||
				       tag == "form" || tag == "/form"

				)
			{
				isEndOfSentenceMarker = true;
				return true;
			}
      
      return false;
    }

    protected void StructsToConsider(string tag, HtmlStructure htmlStructure)
    {
			if (m_structsToConsider == null) return;

      for (int i=0;i<m_structsToConsider.Length;i++)
      {
        Utils.Structure structure = m_structsToConsider[i];
        if (tag == structure.StructureStartTag)
        {
          HtmlStructure importantStructure = 
            new HtmlStructure(structure.StructureName,structure.StructureEndTag,htmlStructure.DepthCounter + 1);
          string table = this.ReadUntilPatternFound(">",'>',FindDirection.Right);
          htmlStructure.Structure.Add(importantStructure);

          //must advance index because current character is '>'
          AdvanceIndex();
          this.ParseStructure(importantStructure);
          break;
        }
      }
    }

    #region Debug functions

    public void WriteStructures(TextWriter writer,HtmlStructure structure)
    {
      if (writer == null) return;
      if (structure == null) return;

      IList structList = structure.Structure;

      writer.WriteLine(structure.TagName + " - " + structure.DepthCounter + " - " + structure.ThisCounter);

			

			for (int i=0;i<structList.Count;i++)
			{
				WriteStructures(writer,(HtmlStructure)structList[i]);
			}
      structure.TextArrayString(writer);
			this.WriteAnchors(writer,structure);

      
    }

    public void WriteMetaNames(TextWriter writer,HtmlHeader header)
    {
      if (writer == null) return;
      if (header == null) return;

      IList metanameList = header.Metaname;
			if (metanameList == null) return;

      for (int i=0;i<metanameList.Count;i++)
      {
       writer.WriteLine(metanameList[i].ToString());
      }
    }

		/// <summary>
		/// Get the metanames as string
		/// </summary>
		/// <returns></returns>
		public string GetMetaNamesString()
		{
			if (this.Header == null) return String.Empty;
			StringBuilder stringBuilder = new StringBuilder();

			stringBuilder.Append(Environment.NewLine);
			stringBuilder.Append(this.Header.Title);
													 
			if (this.Header.Metaname == null || this.Header.Metaname.Count == 0) return stringBuilder.ToString();
			
			foreach(string str in this.Header.Metaname)
			{
				stringBuilder.Append(Environment.NewLine);
				stringBuilder.Append(str);
			}

			return stringBuilder.ToString();
		}

		public string GetAnchorTextString(HtmlStructure structure)
		{
			if (structure.Anchors == null) return String.Empty;

			IList anchorList = structure.Anchors;
			StringBuilder stringBuilder = new StringBuilder();

			for (int i=0;i<anchorList.Count;i++)
			{
				stringBuilder.Append(Environment.NewLine);
				stringBuilder.Append(((HtmlAnchor)anchorList[i]).Text);
			}

			return stringBuilder.ToString();
		}

    public void WriteAnchors(TextWriter writer,HtmlStructure structure)
    {

      if (writer == null) return;

      IList anchorList = structure.Anchors;
      if (anchorList == null) return;

      for (int i=0;i<anchorList.Count;i++)
      {
        writer.WriteLine(((HtmlAnchor)anchorList[i]).Href.ToString() + "\t" + ((HtmlAnchor)anchorList[i]).Text);
      }
    }

    #endregion
   
	}
}
